#!/usr/bin/python
#-*-coding:utf-8-*-
from scrapy import log
from scrapy.exceptions import IgnoreRequest
from gaodeha_crawl.common.db_utils import DbUtils
import random

class FilterRequestMiddleware():
    def __init__(self):
        self.db = DbUtils()

    def process_request(self, request, spider):
        sql = "select count(*) from tb_post where post_origin_url = %s"
        params = (request.url,)
        count = self.db.fetchOneRaw(sql, params)[0]
        if count and count > 0:
            log.msg("ignore request url:" + request.url)
            raise IgnoreRequest(request, spider)

        
        
        

